diff --git a/README b/README
index fefa71c..cac7978 100644
--- a/README
+++ b/README
@@ -3176,8 +3176,13 @@
 		This enables the RSA algorithm used for FIT image verification
 		in U-Boot. See doc/uImage.FIT/signature.txt for more information.
 
+		The Modular Exponentiation algorithm in RSA is implemented using
+		driver model. So CONFIG_DM needs to be enabled by default for this
+		library to function.
+
 		The signing part is build into mkimage regardless of this
-		option.
+		option. The software based modular exponentiation is built into
+		mkimage irrespective of this option.
 
 - bootcount support:
 		CONFIG_BOOTCOUNT_LIMIT
diff --git a/configs/am335x_boneblack_vboot_defconfig b/configs/am335x_boneblack_vboot_defconfig
index 5837a0a..51bf370 100644
--- a/configs/am335x_boneblack_vboot_defconfig
+++ b/configs/am335x_boneblack_vboot_defconfig
@@ -4,3 +4,7 @@
 +S:CONFIG_TARGET_AM335X_EVM=y
 CONFIG_OF_CONTROL=y
 CONFIG_DEFAULT_DEVICE_TREE="am335x-boneblack"
+CONFIG_FIT=y
+CONFIG_FIT_VERBOSE=y
+CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/ids8313_defconfig b/configs/ids8313_defconfig
index 8479cd4..0950ec8 100644
--- a/configs/ids8313_defconfig
+++ b/configs/ids8313_defconfig
@@ -4,3 +4,4 @@
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_TARGET_IDS8313=y
+CONFIG_DM=y
diff --git a/configs/sandbox_defconfig b/configs/sandbox_defconfig
index 0111f25..660063e 100644
--- a/configs/sandbox_defconfig
+++ b/configs/sandbox_defconfig
@@ -3,4 +3,5 @@
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
 CONFIG_DEFAULT_DEVICE_TREE="sandbox"
diff --git a/configs/zynq_microzed_defconfig b/configs/zynq_microzed_defconfig
index b9a6fe5..8b985fe 100644
--- a/configs/zynq_microzed_defconfig
+++ b/configs/zynq_microzed_defconfig
@@ -6,4 +6,5 @@
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
 CONFIG_DEFAULT_DEVICE_TREE="zynq-microzed"
diff --git a/configs/zynq_zc70x_defconfig b/configs/zynq_zc70x_defconfig
index dc8aa84..cceb321 100644
--- a/configs/zynq_zc70x_defconfig
+++ b/configs/zynq_zc70x_defconfig
@@ -7,3 +7,4 @@
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zc770_xm010_defconfig b/configs/zynq_zc770_xm010_defconfig
index 2f5fa8c..2935c0d 100644
--- a/configs/zynq_zc770_xm010_defconfig
+++ b/configs/zynq_zc770_xm010_defconfig
@@ -8,3 +8,4 @@
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zc770_xm012_defconfig b/configs/zynq_zc770_xm012_defconfig
index a92d495..0401739 100644
--- a/configs/zynq_zc770_xm012_defconfig
+++ b/configs/zynq_zc770_xm012_defconfig
@@ -8,3 +8,4 @@
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zc770_xm013_defconfig b/configs/zynq_zc770_xm013_defconfig
index 3a02f75..a95970a 100644
--- a/configs/zynq_zc770_xm013_defconfig
+++ b/configs/zynq_zc770_xm013_defconfig
@@ -8,3 +8,4 @@
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zed_defconfig b/configs/zynq_zed_defconfig
index 1d816f6..0fbc41a 100644
--- a/configs/zynq_zed_defconfig
+++ b/configs/zynq_zed_defconfig
@@ -7,3 +7,4 @@
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zybo_defconfig b/configs/zynq_zybo_defconfig
index 9183629..4e66760 100644
--- a/configs/zynq_zybo_defconfig
+++ b/configs/zynq_zybo_defconfig
@@ -7,3 +7,4 @@
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/include/configs/am335x_evm.h b/include/configs/am335x_evm.h
index f9bc23b..76ce7de 100644
--- a/include/configs/am335x_evm.h
+++ b/include/configs/am335x_evm.h
@@ -19,13 +19,11 @@
 #include <configs/ti_am335x_common.h>
 
 #ifndef CONFIG_SPL_BUILD
+#ifndef CONFIG_FIT
 # define CONFIG_FIT
+#endif
 # define CONFIG_TIMESTAMP
 # define CONFIG_LZO
-# ifdef CONFIG_ENABLE_VBOOT
-# define CONFIG_FIT_SIGNATURE
-# define CONFIG_RSA
-# endif
 #endif
 
 #define CONFIG_SYS_BOOTM_LEN		(16 << 20)
diff --git a/include/configs/sandbox.h b/include/configs/sandbox.h
index 6fd29b9..e9d3f32 100644
--- a/include/configs/sandbox.h
+++ b/include/configs/sandbox.h
@@ -23,7 +23,6 @@
 
 #define CONFIG_BOOTSTAGE
 #define CONFIG_BOOTSTAGE_REPORT
-#define CONFIG_DM
 #define CONFIG_CMD_DEMO
 #define CONFIG_CMD_DM
 #define CONFIG_DM_DEMO
diff --git a/lib/rsa/rsa-verify.c b/lib/rsa/rsa-verify.c
index f8bc086..da45daf 100644
--- a/lib/rsa/rsa-verify.c
+++ b/lib/rsa/rsa-verify.c
@@ -12,6 +12,7 @@
 #include <asm/errno.h>
 #include <asm/types.h>
 #include <asm/unaligned.h>
+#include <dm.h>
 #else
 #include "fdt_host.h"
 #include "mkimage.h"
@@ -43,6 +44,9 @@
 	const uint8_t *padding;
 	int pad_len;
 	int ret;
+#if !defined(USE_HOSTCC)
+	struct udevice *mod_exp_dev;
+#endif
 
 	if (!prop || !sig || !hash || !algo)
 		return -EIO;
@@ -63,7 +67,17 @@
 
 	uint8_t buf[sig_len];
 
+#if !defined(USE_HOSTCC)
+	ret = uclass_get_device(UCLASS_MOD_EXP, 0, &mod_exp_dev);
+	if (ret) {
+		printf("RSA: Can't find Modular Exp implementation\n");
+		return -EINVAL;
+	}
+
+	ret = rsa_mod_exp(mod_exp_dev, sig, sig_len, prop, buf);
+#else
 	ret = rsa_mod_exp_sw(sig, sig_len, prop, buf);
+#endif
 	if (ret) {
 		debug("Error in Modular exponentation\n");
 		return ret;